jobs:
  - ${{ each topic in parameters.domains }}:
    - job:
      displayName: "domain ${{topic}} with 2 GPU"
      # how long to run the job before automatically cancelling
      timeoutInMinutes: "45"
      # how much time to give 'run always even if cancelled tasks' before stopping them
      cancelTimeoutInMinutes: "2"

      pool: "lit-rtx-3090"
      variables:
        DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )

      # this need to have installed docker in the base image...
      container:
        # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04
        # image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.11"
        image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime"
        options: "-it --rm --gpus=all --shm-size=16g -v /usr/bin/docker:/tmp/docker:ro"

      workspace:
        clean: all
      steps:

      - bash: |
          echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
          echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
        displayName: 'Set environment variables'

      - bash: |
          echo $CUDA_VISIBLE_DEVICES
          lspci | egrep 'VGA|3D'
          whereis nvidia
          nvidia-smi
          pip --version
          pip list
          df -kh /dev/shm
        displayName: 'Image info & NVIDIA'

      - script: |
          python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
        displayName: 'Sanity check'

      - script: |
          python -m pip install "pip==22.2.1"
          pip --version
          pip install '.[${{topic}},test]' -r "requirements/testing_${{topic}}.txt" -U --prefer-binary
        env:
          FREEZE_REQUIREMENTS: 1
        displayName: 'Install dependencies'

      - script: |
          pip list
          python -m coverage run --source flash -m pytest tests/examples -vV --timeout=300 --durations=30
        displayName: 'Testing'

      - bash: |
          python -m coverage report
          python -m coverage xml
          # python -m codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) --flags=gpu,pytest --name="GPU-coverage" --env=linux,azure
          ls -l
        displayName: 'Statistics'
